"""
Author: Yishang
Description: 用于提取网页的特征、内容等。
"""
import re

from bs4 import BeautifulSoup

from urllib.parse import urljoin
from typing import List, Tuple, Union, Callable



from urllib.parse import urljoin
from typing import List, Union, Callable, Tuple
from bs4 import BeautifulSoup
import re

def get_links(
    soup: BeautifulSoup,
    url: str,
    whitelist: List[Union[str, Callable[[str], bool]]] = (),
    blacklist: List[Union[str, Callable[[str], bool]]] = ()
) -> List[Tuple[str, BeautifulSoup]]:
    """
    提取网页中的链接，返回补全后的绝对 URL 列表

    :param url: 用于补全相对路径的基地址 (e.g. "http://example.com/path/page.html")
    """
    links = []

    for tag in soup.find_all():
        # 提取 href 或 src 属性
        link = None
        if tag.has_attr("href"):
            link = tag["href"]
        elif tag.has_attr("src"):
            link = tag["src"]

        if not link:
            continue

        # 跳过空链接和 JavaScript 伪协议
        link = link.strip()
        if not link or link.startswith("javascript:"):
            continue

        # 补全为绝对 URL
        full_url = urljoin(url, link)

        # 检查黑名单
        blacklisted = False
        for pattern in blacklist:
            if isinstance(pattern, str) and re.search(pattern, full_url):
                blacklisted = True
                break
            elif callable(pattern) and pattern(full_url):
                blacklisted = True
                break
        if blacklisted:
            continue

        # 检查白名单
        whitelisted = False
        if not whitelist:  # 白名单为空时默认全部允许
            whitelisted = True
        else:
            for pattern in whitelist:
                if isinstance(pattern, str) and re.search(pattern, full_url):
                    whitelisted = True
                    break
                elif callable(pattern) and pattern(full_url):
                    whitelisted = True
                    break

        if whitelisted:
            links.append(full_url)

    return links